import java.io.*;
import java.net.*;
import java.util.*;
import java.util.regex.*;
import java.net.URLEncoder;

public class Main {

    public static String[] wikiURL = {
                       "http://ja.wikipedia.org/wiki/AV%E5%A5%B3%E5%84%AA%E4%B8%80%E8%A6%A7_%E3%81%82%E8%A1%8C",
                       "http://ja.wikipedia.org/wiki/AV%E5%A5%B3%E5%84%AA%E4%B8%80%E8%A6%A7_%E3%81%8B%E8%A1%8C",
                       "http://ja.wikipedia.org/wiki/AV%E5%A5%B3%E5%84%AA%E4%B8%80%E8%A6%A7_%E3%81%95%E8%A1%8C",
                       "http://ja.wikipedia.org/wiki/AV%E5%A5%B3%E5%84%AA%E4%B8%80%E8%A6%A7_%E3%81%9F%E8%A1%8C",
                       "http://ja.wikipedia.org/wiki/AV%E5%A5%B3%E5%84%AA%E4%B8%80%E8%A6%A7_%E3%81%AA%E8%A1%8C",
                       "http://ja.wikipedia.org/wiki/AV%E5%A5%B3%E5%84%AA%E4%B8%80%E8%A6%A7_%E3%81%AF%E8%A1%8C",
                       "http://ja.wikipedia.org/wiki/AV%E5%A5%B3%E5%84%AA%E4%B8%80%E8%A6%A7_%E3%81%BE%E8%A1%8C",
                       "http://ja.wikipedia.org/wiki/AV%E5%A5%B3%E5%84%AA%E4%B8%80%E8%A6%A7_%E3%82%84%E8%A1%8C",
                       "http://ja.wikipedia.org/wiki/AV%E5%A5%B3%E5%84%AA%E4%B8%80%E8%A6%A7_%E3%82%89%E3%83%BB%E3%82%8F%E8%A1%8C"
    };

    public static void main(String[] args)
    {
        try {
            scrapeWiki();
        } catch (MalformedURLException e){
            System.err.println("MalformedURLException:" + e.getMessage());
        } catch (Exception e){
            System.err.println("Exception:" + e.getMessage());
        }
    }
    
    public static void scrapeWiki()
        throws MalformedURLException, ProtocolException, IOException {
        PrintWriter outFileMain = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream("AVdic.txt"),"UTF-8")));
        PrintWriter outFileAlready = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream("in_dic.txt"),"UTF-8")));
        PrintWriter outFileMuhenkan = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream("muhenkan_dic.txt"),"UTF-8")));
        for(String url: wikiURL){
            List<String> lines = httpGet(url);
            Pattern pa1 = Pattern.compile("^<li><a href=\".+?>([^（）]+).*</a>（(.+?)）.+</li>$");
            Pattern pa2 = Pattern.compile("^[A-Za-z0-9@。\\p{InHiragana}\\p{InKatakana}\\.]+$");
            PrintStream out = new PrintStream(System.out, true, "UTF-8");
            for(String line: lines){
                Matcher matcher = pa1.matcher(line);
                if(matcher.find()){
                    String henkango = matcher.group(1).replaceAll("&amp;", "&");
                    String yomi = matcher.group(2).replaceAll(" ", "").replaceAll("&amp;", "&");
                    Matcher ma2 = pa2.matcher(henkango);
                    if (ma2.find()){
                        outFileMuhenkan.println(yomi + "\t" + henkango + "\t人名");
                    } else {
                        String g = searchGoogleCGI("http://www.google.com/transliterate?langpair=ja-Hira|ja&text=" + URLEncoder.encode(yomi + ",", "UTF-8"));
                        if (g.indexOf(henkango) != -1){
                            outFileAlready.println(yomi + "\t" + henkango + "\t人名");
                        } else {
                            outFileMain.println(yomi + "\t" + henkango + "\t人名");
                        }
                    }
                }
            }
        }
        outFileMain.close(); 
        outFileAlready.close(); 
        outFileMuhenkan.close(); 
    }
    
    public static String searchGoogleCGI(String s)
        throws MalformedURLException, ProtocolException, IOException {
            PrintStream out = new PrintStream(System.out, true, "UTF-8");
            List<String> lines = httpGet(s);
            for(String line: lines){
                return line;
            }
            return null;
        }
}
